{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 前言\n",
    "\n",
    "优化问题是金融中基础、不可避免的问题，从均值方差的二次规划开始， 优化问题已经深入到金融领域的方方面面，从大类资产配置到组合优化、从理论模型下的效用最大化再到实战模型的参数优化，都用到优化技术。而很多优化问题较为复杂，非凸、不连续、不可导、高维、随机、约束过多等问题给数 值计算带来困扰，本文提出**次优理论**并且介绍差分进化算法，通过展示差分进 化算法的良好效果，希望给广大投资者的量化建模带来一丝启示。本文方法对组合优化、大类配置、FOF组合构建、智能投顾等领域都会有所帮助。\n",
    "\n",
    "**当期优化的最优解不一定是下一期的最优，而样本内的次优在样本外可能战胜样本内的最优**。所以，从金融投资角度看，优化问题下的最优解不一定是我们想要的，因为我们的目标是获得较好的样本外收益表现。\n",
    "\n",
    "(https://www.joinquant.com/view/community/detail/2044ade4baf51132d257f2d3c0e56597)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'empyrical'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Input \u001b[1;32mIn [2]\u001b[0m, in \u001b[0;36m<cell line: 6>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mempyrical\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mep\u001b[39;00m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mscipy\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mstats\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mst\u001b[39;00m\n",
      "\u001b[1;31mModuleNotFoundError\u001b[0m: No module named 'empyrical'"
     ]
    }
   ],
   "source": [
    "# 初始库\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import empyrical as ep\n",
    "import scipy.stats as st"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "from DE_algorithm import *\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "from typing import (Tuple,List)\n",
    "from dateutil.parser import parse\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('ggplot')\n",
    "\n",
    "from jqdata import *\n",
    "\n",
    "# 设置字体 用来正常显示中文标签\n",
    "#plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.rcParams['font.family']='serif'\n",
    "\n",
    "# 用来正常显示负号\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "\n",
    "# 图表主题\n",
    "plt.style.use('seaborn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "code_folding": [
     1,
     33,
     61,
     81,
     114
    ]
   },
   "outputs": [],
   "source": [
    "# 构造时间区间\n",
    "def build_periods(start:str, end:str, freq:str,offsetN:int=None) -> Tuple[list, list]:\n",
    "    '''构造区间'''\n",
    "\n",
    "    # 半年度要特殊一点\n",
    "    if freq == 'half':\n",
    "\n",
    "        periods = [i.strftime(\n",
    "            '%Y-%m-%d') for i in pd.date_range(start, end, freq='Q') if i.month in [6, 12]]\n",
    "\n",
    "        end_range = periods[1:] + [end]\n",
    "        start_range = periods\n",
    "\n",
    "    else:\n",
    "\n",
    "        periods = [i.strftime('%Y-%m-%d')\n",
    "                   for i in pd.date_range(start, end, freq=freq)]\n",
    "\n",
    "        start_range = periods[:-1]\n",
    "        end_range = periods[1:]\n",
    "\n",
    "    if offsetN:\n",
    "\n",
    "        start_range = [tdaysoffset(i, offsetN) for i in start_range]\n",
    "        end_range = [tdaysoffset(i, offsetN) for i in end_range]\n",
    "\n",
    "        return (start_range, end_range)\n",
    "\n",
    "    else:\n",
    "\n",
    "        return (start_range, end_range)\n",
    "\n",
    "# offset day\n",
    "def tdaysoffset(end_date: str, count: int) -> datetime.date:\n",
    "    '''\n",
    "    end_date:为基准日期\n",
    "    count:为正则后推，负为前推\n",
    "    -----------\n",
    "    return datetime.date\n",
    "    '''\n",
    "\n",
    "    trade_date = get_trade_days(end_date=end_date, count=1)[0]\n",
    "\n",
    "    if count > 0:\n",
    "        # 将end_date转为交易日\n",
    "\n",
    "        trade_cal = get_all_trade_days().tolist()\n",
    "\n",
    "        trade_idx = trade_cal.index(trade_date)\n",
    "\n",
    "        return trade_cal[trade_idx + count]\n",
    "\n",
    "    elif count < 0:\n",
    "\n",
    "        return get_trade_days(end_date=trade_date, count=abs(count))[0]\n",
    "\n",
    "    else:\n",
    "\n",
    "        raise ValueError('别闹！')\n",
    "        \n",
    "# 风险指标\n",
    "def Strategy_performance(return_df: pd.DataFrame, periods='monthly') -> pd.DataFrame:\n",
    "    '''计算风险指标 默认为月度:月度调仓'''\n",
    "\n",
    "    ser: pd.DataFrame = pd.DataFrame()\n",
    "    ser['年化收益率'] = ep.annual_return(return_df, period=periods)\n",
    "    ser['波动率'] = return_df.apply(lambda x: ep.annual_volatility(x,period=periods))\n",
    "    ser['夏普'] = return_df.apply(ep.sharpe_ratio, period=periods)\n",
    "    ser['最大回撤'] = return_df.apply(lambda x: ep.max_drawdown(x))\n",
    "    \n",
    "    if 'benchmark' in return_df.columns:\n",
    "\n",
    "        select_col = [col for col in return_df.columns if col != 'benchmark']\n",
    "\n",
    "        ser['IR'] = return_df[select_col].apply(\n",
    "            lambda x: information_ratio(x, return_df['benchmark']))\n",
    "        ser['Alpha'] = return_df[select_col].apply(\n",
    "            lambda x: ep.alpha(x, return_df['benchmark'], period=periods))\n",
    "\n",
    "    return ser.T\n",
    "\n",
    "def information_ratio(returns, factor_returns):\n",
    "    \"\"\"\n",
    "    Determines the Information ratio of a strategy.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    returns : :py:class:`pandas.Series` or pd.DataFrame\n",
    "        Daily returns of the strategy, noncumulative.\n",
    "        See full explanation in :func:`~empyrical.stats.cum_returns`.\n",
    "    factor_returns: :class:`float` / :py:class:`pandas.Series`\n",
    "        Benchmark return to compare returns against.\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    :class:`float`\n",
    "        The information ratio.\n",
    "\n",
    "    Note\n",
    "    -----\n",
    "    See https://en.wikipedia.org/wiki/information_ratio for more details.\n",
    "\n",
    "    \"\"\"\n",
    "    if len(returns) < 2:\n",
    "        return np.nan\n",
    "\n",
    "    active_return = _adjust_returns(returns, factor_returns)\n",
    "    tracking_error = np.std(active_return, ddof=1)\n",
    "    if np.isnan(tracking_error):\n",
    "        return 0.0\n",
    "    if tracking_error == 0:\n",
    "        return np.nan\n",
    "    return np.mean(active_return) / tracking_error\n",
    "\n",
    "def _adjust_returns(returns, adjustment_factor):\n",
    "    \"\"\"\n",
    "    Returns a new :py:class:`pandas.Series` adjusted by adjustment_factor.\n",
    "    Optimizes for the case of adjustment_factor being 0.\n",
    "\n",
    "    Parameters\n",
    "    ----------\n",
    "    returns : :py:class:`pandas.Series`\n",
    "    adjustment_factor : :py:class:`pandas.Series` / :class:`float`\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    :py:class:`pandas.Series`\n",
    "    \"\"\"\n",
    "    if isinstance(adjustment_factor, (float, int)) and adjustment_factor == 0:\n",
    "        return returns.copy()\n",
    "    return returns - adjustment_factor\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [
     1
    ]
   },
   "outputs": [],
   "source": [
    "# 用于回测\n",
    "def BackTesting(timeRange:List,weight:dict)->pd.DataFrame:\n",
    "    '''\n",
    "    timeRange:时间区间\n",
    "    wieght:权重key-trade,values:pd.Series\n",
    "    ----------------\n",
    "        return 等权、优化权重、基准的收益率序列\n",
    "    '''\n",
    "    \n",
    "    begin = min(timeRange[0])\n",
    "    end = max(timeRange[1])\n",
    "    idx = pd.to_datetime(get_trade_days(begin,end))\n",
    "    # 缓存容器\n",
    "    ret_df = pd.DataFrame(data=np.zeros((len(idx),2)),\n",
    "                          index=idx,\n",
    "                          columns=['opt_ret','equal_ret'])\n",
    "\n",
    "    # 模拟回测\n",
    "    ## 季度:beginDt日买入,至nextDt轮动\n",
    "    for beginDt,nextDt in tqdm_notebook(zip(*timeRange),total=len(timeRange[0])):\n",
    "\n",
    "        # 权重优化基于beginDt\n",
    "        weight_ser = weight[beginDt] # 读取权重\n",
    "        code = weight_ser.index.tolist()\n",
    "\n",
    "        # 持仓从beginDt日至nextDt日\n",
    "        price = get_price(code,beginDt,tdaysoffset(nextDt,1),fields='close',panel=False)\n",
    "        close = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "        pct_df = close.pct_change().shift(-1).iloc[:-1] # 收益滞后一期 防止未来数据\n",
    "\n",
    "        ret_df.loc[pct_df.index,'opt_ret'] = pct_df @ weight_ser # 计算组合权重\n",
    "        ret_df.loc[pct_df.index,'equal_ret'] = pct_df.mean(axis=1) # 等权\n",
    "        \n",
    "    benchmark = get_price('000300.XSHG',startDt,endDt,fields='close')\n",
    "    benchmark = benchmark.pct_change().reindex(ret_df.index)\n",
    "    ret_df['benchmark'] = benchmark\n",
    "    return ret_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DE算法权重优化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "差分进化算法参数是个技术活...."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import date, timedelta, datetime as dt\n",
    "startDt = date.today() + timedelta(days = -2193)\n",
    "endDt = date.today() + timedelta(days = -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 设置标的及时间范围\n",
    "# 设置时间范围\n",
    "# startDt = '2014-01-01'\n",
    "# endDt = '2019-12-31'\n",
    "\n",
    "# 设置调仓频率为季度\n",
    "timeRange = build_periods(startDt,endDt,'Q')\n",
    "\n",
    "# 设置标的\n",
    "etfList = '510050.XSHG,510500.XSHG,510300.XSHG,159915.XSHE,510180.XSHG,159901.XSHE,159902.XSHE'.split(',')\n",
    "\n",
    "# 测试数据获取\n",
    "price = get_price(etfList,end_date = timeRange[0][0],count=126,fields='close',panel=False)\n",
    "\n",
    "price = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "\n",
    "pct_chg = price.pct_change().iloc[1:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 超参数\n",
    "\n",
    "寻找在de进化算法下的最优初始参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "from sklearn.base import TransformerMixin, BaseEstimator\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from typing import (Callable)\n",
    "\n",
    "'''使用sklearn接口用于超参'''\n",
    "class DEPortfolioOpt(BaseEstimator):\n",
    "    \n",
    "    def __init__(self,func:Callable,size_pop:int,max_iter:int,F:float,proub_mut:float)->None:\n",
    "        \n",
    "        self.func = func           # 目标函数\n",
    "        self.size_pop = size_pop   # 种群大小\n",
    "        self.max_iter = max_iter   # 迭代次数\n",
    "        self.F = F                 # 变异系数\n",
    "        self.proub_mut = proub_mut # 变异概率\n",
    "        \n",
    "    def fit(self,returns:pd.DataFrame)->np.array:\n",
    "        \n",
    "        '''获取优化后的权重'''\n",
    "        \n",
    "        self.de = DE(func= lambda w:self.func(returns,w),\n",
    "               n_dim = returns.shape[1],\n",
    "               size_pop = self.size_pop,\n",
    "               max_iter = self.max_iter,\n",
    "               F=self.F,\n",
    "               prob_mut = self.proub_mut,\n",
    "               lb=[0],\n",
    "               ub=[1],\n",
    "               constraint_eq=[lambda x:1 - np.sum(x)])\n",
    "        \n",
    "        w,self.maxDrawdown = self.de.run()\n",
    "        \n",
    "        return pd.Series(w,index=returns.columns)\n",
    "    \n",
    "    def predict(self,returns)->float:\n",
    "        '''获取权重后的收益率'''\n",
    "        w = self.fit(returns)\n",
    "        return returns @ w\n",
    "    \n",
    "    def score(self,returns)->float:\n",
    "        '''评分根据优化后的回撤决定'''\n",
    "        return -self.maxDrawdown\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def maxDrawdown(pct_chg:pd.DataFrame,w:np.array)->float:\n",
    "    '''目标函数'''\n",
    "    return - ep.max_drawdown(pct_chg @ w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 网格超参\n",
    "## 等待时间有点漫长大概1h+\n",
    "param_grid = [{'F':np.arange(0.2,1.1,0.1),\n",
    "              'proub_mut':np.arange(0.1,1.1,0.1)}]\n",
    "\n",
    "\n",
    "de_portfolio = DEPortfolioOpt(maxDrawdown,150,250,0.5,1)\n",
    "\n",
    "# cv默认为3flod\n",
    "grid_search = GridSearchCV(de_portfolio,param_grid)\n",
    "\n",
    "grid_search.fit(pct_chg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "最优参数为F:0.5,proub_mut:1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 最优参数\n",
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "equal = -ep.max_drawdown(pct_chg.mean(axis=1))\n",
    "print('等权回撤:%.4f,优化回撤:%.4f'%(equal,- grid_search.best_estimator_.score(pct_chg)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到在迭代到200次时，曲线基本上没有什么变化了,所有将max_iter设置在200就差不多了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 回撤情况    \n",
    "maxdrawdownList = [-ep.max_drawdown(pct_chg @ w) for w in grid_search.best_estimator_.de.generation_best_X]\n",
    "\n",
    "plt.rcParams['font.family']='serif'\n",
    "fig,axes = plt.subplots(1,2,figsize=(6 * 2,4))\n",
    "\n",
    "axes[0].set_title('拟合曲线')\n",
    "axes[0].plot(maxdrawdownList,label='拟合的回撤')\n",
    "axes[0].axhline(equal,color='r',label='等权回撤')\n",
    "\n",
    "axes[1].set_title('拟合度变化')\n",
    "axes[1].plot(grid_search.best_estimator_.de.generation_best_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_x = grid_search.best_estimator_.de.best_x # 最优组合\n",
    "\n",
    "price = get_price(etfList,start_date=timeRange[0][0],end_date = timeRange[0][1],fields='close',panel=False)\n",
    "\n",
    "price = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "\n",
    "pct_chg = price.pct_change().iloc[1:]\n",
    "\n",
    "print('样本外回撤情况%.4f(等权:%.4f)'%(ep.max_drawdown(pct_chg @ best_x),ep.max_drawdown(pct_chg.mean(axis=1))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "红线左侧为样本内优化情况,右侧为样本外情况，可以看到样本外并不好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "price = get_price(etfList,\n",
    "                  start_date=tdaysoffset(timeRange[0][0],-60),\n",
    "                  end_date=timeRange[0][1],\n",
    "                  fields='close',\n",
    "                  panel=False)\n",
    "\n",
    "price = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "\n",
    "pct_chg = price.pct_change().iloc[1:]\n",
    "# 样本内\n",
    "in_sample = pct_chg.loc[:timeRange[0][0]]\n",
    "# 样本外\n",
    "out_of_sample = pct_chg.loc[timeRange[0][0]:]\n",
    "\n",
    "fig,ax = plt.subplots(figsize=(18,4))\n",
    "ax.set_title('权重优化对比')\n",
    "\n",
    "ep.cum_returns(in_sample @ best_x).plot(ax=ax,label='in_sample',color='r')\n",
    "ep.cum_returns(out_of_sample @ best_x).plot(ax=ax,label='out_of_sample',color='r')\n",
    "ep.cum_returns(out_of_sample.mean(axis=1)).plot(ax=ax,label='equal',color='darkgray')\n",
    "ep.cum_returns(in_sample.mean(axis=1)).plot(ax=ax,label='equal',color='darkgray')\n",
    "plt.axvline(timeRange[0][0],color='black')\n",
    "\n",
    "h1,l1 = ax.get_legend_handles_labels()\n",
    "h1 = h1[:-1]\n",
    "l1 = l1[:-1]\n",
    "plt.legend(h1,l1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**回测逻辑:**\n",
    "\n",
    "在每季度计算标的的最优权重(使用前125日的数据),持有至下一个季度换仓。\n",
    "\n",
    "即在T-1日收盘后，使用T-1至T-(1-N)日收益率序列进行组合优化，在T日按照T-1日的优化结果持有对应的标的,直至下一个换仓日。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# 计算权重\n",
    "def get_weight(opt:DEPortfolioOpt,codes:list,watch_date:str,N:int,)->pd.Series:\n",
    "    \n",
    "    '''获取组合权重'''\n",
    "    \n",
    "    price = get_price(codes,end_date=watch_date,count=N,fields='close',panel=False)\n",
    "    close = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "    \n",
    "    # 收益滞后一期 去除头尾的na值\n",
    "    pct_df = close.pct_change().iloc[1:]\n",
    "    \n",
    "    return opt.fit(pct_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每期的权重\n",
    "'''\n",
    "每季度获取前125日的数据进行组合优化\n",
    "在观察日的基础上前移一天避免未来数据\n",
    "'''\n",
    "\n",
    "etf_weight_dic = {tradeDt: get_weight(DEPortfolioOpt(maxDrawdown,150,200,0.5,1), etfList, tdaysoffset(\n",
    "    tradeDt, -2), 126) for tradeDt in timeRange[0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 回测\n",
    "returns_df = BackTesting(timeRange,etf_weight_dic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "每一期组合优化回撤与等权回撤对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 查看优化的组合回撤情况\n",
    "\n",
    "max_drawdown_df = pd.DataFrame(index=pd.to_datetime(timeRange[0])\n",
    "                               ,columns=['opt_maxDrawDown','equal_maxDrawDown'])\n",
    "\n",
    "for s,e in zip(*timeRange):\n",
    "    \n",
    "    \n",
    "    max_drawdown_df.loc[s,'opt_maxDrawDown'] = - ep.max_drawdown(returns_df.loc[s:e,'opt_ret'])\n",
    "    max_drawdown_df.loc[s,'equal_maxDrawDown'] = - ep.max_drawdown(returns_df.loc[s:e,'equal_ret'])\n",
    "    \n",
    "plt.rcParams['font.family']='serif'\n",
    "max_drawdown_df.plot.bar(y=['opt_maxDrawDown','equal_maxDrawDown'],\n",
    "                         color=['r','darkgray'],\n",
    "                         title='优化效果',\n",
    "                        figsize=(12,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 最大回撤（MD）最小化：\n",
    "\n",
    "**目标函数**\n",
    "\n",
    "1. 最大回撤(MD)最小\n",
    "\n",
    "\n",
    "\n",
    "$$\n",
    "\\begin{matrix}\n",
    "    min\\ maxDrawdown(\\sum^{}_{}w_ir_i) \\\\\n",
    "    subject\\ to\\ \\sum^{}_{}w_i=1 \\\\\n",
    "                   0\\leq w_i\\leq 1\\\\\n",
    "\\end{matrix}\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ETF标的池一"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "有一定的优化效果,但感觉效果并不是特别好。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "cumRet = 1 + ep.cum_returns(returns_df)\n",
    "\n",
    "cumRet.plot.line(y=['opt_ret','equal_ret','benchmark'],\n",
    "                 label=['MP_pool','equal','沪深300'],\n",
    "                 color=['r','#6891BD','darkgray'],\n",
    "                 figsize=(18,4),\n",
    "                title='MD ETF Pool 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 风险指标s\n",
    "Strategy_performance(returns_df,'daily').style.format('{:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ETF标的池二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "etfList2 = ('510050.XSHG,510500.XSHG,510300.XSHG,159915.XSHE,510180.XSHG,159901.XSHE,'\n",
    "            '510230.XSHG,159928.XSHE,510880.XSHG,159938.XSHE,159902.XSHE,512070.XSHG,'\n",
    "            '159939.XSHE,159905.XSHE,159910.XSHE').split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每期的权重\n",
    "# 每季度获取前125日的数据进行组合优化\n",
    "# 在观察日的基础上前移一天避免未来数据\n",
    "etf_weight_dic2 = {tradeDt: get_weight(DEPortfolioOpt(maxDrawdown,150,200,0.5,1), etfList2, tdaysoffset(\n",
    "    tradeDt, -2), 126) for tradeDt in timeRange[0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "returns_df = BackTesting(timeRange,etf_weight_dic2)\n",
    "cumRet = 1 + ep.cum_returns(returns_df)\n",
    "\n",
    "cumRet.plot.line(y=['opt_ret','equal_ret','benchmark'],\n",
    "                 label=['MP_pool','equal','沪深300'],\n",
    "                 color=['r','#6891BD','darkgray'],\n",
    "                 figsize=(18,4),\n",
    "                title='MD ETF Pool 2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 风险指标\n",
    "Strategy_performance(returns_df,'daily').style.format('{:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 预期损失（CVaR）最小化\n",
    "\n",
    "**目标函数**\n",
    "\n",
    "2. 预期损失(CVaR)最小化\n",
    "\n",
    "<font size=4.5>$$min\\ \\frac{w_{i}\\big[-\\mu_{i}+\\frac{(\\sum{w})_{i}\\phi(Z_{\\alpha})}{\\sqrt{w'\\Sigma w}\\alpha}\\big]}{-w'\\mu+\\sqrt{w'\\Sigma w}\\frac{\\phi(Z_{\\alpha})}{\\alpha}}$$</font>\n",
    "\n",
    "$$subject \\sum{w_{i}}=1$$\n",
    "\n",
    "$$0\\leq w_{i} \\leq 1 $$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     1
    ]
   },
   "outputs": [],
   "source": [
    "# CVaR\n",
    "def calcCVaR(returns:pd.DataFrame,w:np.array,alpha:float=0.05)->float:\n",
    "    '''目标函数'''\n",
    "    #w = np.ones(pct_chg.shape[1]) / pct_chg.shape[1] # 组合权重-等权\n",
    "    N = len(returns)\n",
    "    p_mean = (returns @ w).mean() # 组合收益平均值 \n",
    "    p_var = (returns @ w).var()   # 组合方差\n",
    "    p_std = np.sqrt(p_var)        # 组合标准差\n",
    "\n",
    "    p_std_h = p_std * np.sqrt(N / 252)\n",
    "\n",
    "    CVaR = alpha ** (-1) * st.norm.pdf(st.norm.ppf(alpha)) * p_std_h - p_mean\n",
    "    \n",
    "    return CVaR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ETF标的池一\n",
    "\n",
    "回测结果与研报相当"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每期的权重\n",
    "# 每季度获取前125日的数据进行组合优化\n",
    "# 在观察日的基础上前移一天避免未来数据\n",
    "etf_weight_dic3 = {tradeDt: get_weight(DEPortfolioOpt(calcCVaR,150,200,0.5,1), etfList, tdaysoffset(\n",
    "    tradeDt, -2), 126) for tradeDt in timeRange[0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "returns_df = BackTesting(timeRange,etf_weight_dic3)\n",
    "cumRet = 1 + ep.cum_returns(returns_df)\n",
    "\n",
    "cumRet.plot.line(y=['opt_ret','equal_ret','benchmark'],\n",
    "                 label=['MP_pool','equal','沪深300'],\n",
    "                 color=['r','#6891BD','darkgray'],\n",
    "                 figsize=(18,4),\n",
    "                title='CVaR ETF Pool 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 风险指标\n",
    "Strategy_performance(returns_df,'daily').style.format('{:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ETF标的池二"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每期的权重\n",
    "# 每季度获取前125日的数据进行组合优化\n",
    "# 在观察日的基础上前移一天避免未来数据\n",
    "etf_weight_dic4 = {tradeDt: get_weight(DEPortfolioOpt(calcCVaR,150,200,0.5,1), etfList2, tdaysoffset(\n",
    "    tradeDt, -2), 126) for tradeDt in timeRange[0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "returns_df = BackTesting(timeRange,etf_weight_dic4)\n",
    "cumRet = 1 + ep.cum_returns(returns_df)\n",
    "\n",
    "cumRet.plot.line(y=['opt_ret','equal_ret','benchmark'],\n",
    "                 label=['MP_pool','equal','沪深300'],\n",
    "                 color=['r','#6891BD','darkgray'],\n",
    "                 figsize=(18,4),\n",
    "                title='CVaR ETF Pool 2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 风险指标\n",
    "Strategy_performance(returns_df,'daily').style.format('{:.2%}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在ETF基金池二中的效果不太理想,查看F=0.5,prob_mut=1,max_iter=200时的拟合情况，由下图可以看到200左右数值都还有较大的变动,在250至300次时,数据任然在等权的回撤之上。可能需要重新调整参数才能达到较好的优化效果。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "price = get_price(etfList2,end_date = timeRange[0][0],count=126,fields='close',panel=False)\n",
    "\n",
    "price = pd.pivot_table(price,index='time',columns='code',values='close')\n",
    "\n",
    "pct_chg = price.pct_change().iloc[1:]\n",
    "\n",
    "cvar = DEPortfolioOpt(calcCVaR,150,300,0.5,1)\n",
    "cvar.fit(pct_chg)\n",
    "\n",
    "equal = -ep.max_drawdown(pct_chg.mean(axis=1))\n",
    "\n",
    "# 回撤情况    \n",
    "maxdrawdownList = [-ep.max_drawdown(pct_chg @ w) for w in cvar.de.generation_best_X]\n",
    "\n",
    "plt.rcParams['font.family']='serif'\n",
    "fig,axes = plt.subplots(1,2,figsize=(6 * 2,4))\n",
    "\n",
    "axes[0].set_title('拟合曲线')\n",
    "axes[0].plot(maxdrawdownList,label='拟合的回撤')\n",
    "axes[0].axhline(equal,color='r',label='等权回撤')\n",
    "\n",
    "axes[1].set_title('拟合度变化')\n",
    "axes[1].plot(cvar.de.generation_best_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "MarkDown菜单",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
